const puppeteer = require('puppeteer');
const fs = require('fs');

let totalBlogs = [];

async function main() {
  let totalBlogs = [];
  const url = 'https://segmentfault.com/weekly';

  const browser = await puppeteer.launch({headless: true, slowMo: 0});
  const page = (await browser.pages())[0];
  await page.setViewport({width: 1500, height: 0});
  page.on('console', msg => {
    console.log(msg.text());
  });

  page.on('pageerror', msg => {
    console.log(msg);
  });


  let source = '思否' + (new Date(+new Date() + 1000 * 3600 * 8)).toISOString().split('T')[0];

  await page.goto(url, {waitUntil: 'domcontentloaded'});
  await page.waitFor('#page > div.c-d.c-d-e > div.Zd-p-Sc > div:nth-child(1) > div.cc-dc-c > div > div.jc-c > table > tbody > tr');

  totalBlogs = await page.$$eval('#page > div.c-d.c-d-e > div.Zd-p-Sc > div:nth-child(1) > div.cc-dc-c > div > div.jc-c > table > tbody > tr', async (eles, source) => {
    let tmpBlogs = [];
    eles.forEach(async el => {
      let title = el.querySelector('td.al > a').innerText;
      let content = el.querySelector('td.al > a').href;
      tmpBlogs.push({
        title: title,
        content: content,
        original_url: content,
        tab: 'tech',
        sub_tab: '思否',
        source: source,
        author_id: '5f743a5cea022430046c0d7d',
        author_name: '思否',
        author_avatar: '/public/images/segmentfault.png'
      });
    });
    return tmpBlogs;
  }, source);

  // 替换真实content和original_url
  for (let i = 0; i < totalBlogs.length; i++) {
    let blog = totalBlogs[i];
    await page.waitFor((Math.floor(Math.random() * 3) + 1) * 1000);
    await page.goto(blog.content, {waitUntil: 'domcontentloaded'});
    blog.content = blog.original_url = page.url();
  }

  console.log(totalBlogs);
};


main();


